docs/source/tutorials/Preprocessing SIM.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# SIM Metadata Information"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pysus.online_data.SIM import get_CID9_table, get_CID10_table, get_municipios, get_ocupations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-09-15 18:51:51.424 | DEBUG    | pysus.online_data.SIM:get_CID9_table:139 - Stablishing connection with ftp.datasus.gov.br.\n",
      "220 Microsoft FTP Service\n",
      "2023-09-15 18:51:51.448 | DEBUG    | pysus.online_data.SIM:get_CID9_table:143 - Changing FTP work dir to: /dissemin/publicos/SIM/CID9/TABELAS\n",
      "2023-09-15 18:51:51.450 | INFO     | pysus.online_data.SIM:get_CID9_table:153 - Local parquet file found at /home/bida/pysus/SIM_CID9_.parquet\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>DESCRICAO</th>\n",
       "      <th>CAUSAS</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>001  COLERA</td>\n",
       "      <td>001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>002  FEBRES TIFOIDE E PARATIFOIDE</td>\n",
       "      <td>002</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>003  OUTRAS INFECCOES POR SALMONELLA</td>\n",
       "      <td>003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>004  SHIGUELOSE</td>\n",
       "      <td>004</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>005  OUT INTOXIC ALIMENTARES (BACTERIANAS)</td>\n",
       "      <td>005</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>881</th>\n",
       "      <td>E995  LES OUT MEIOS OU N ESP OP GUERRA CONVENC</td>\n",
       "      <td>995</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>882</th>\n",
       "      <td>E996  LES ARMA NUCLEAR EM OPERACOES DE GUERRA</td>\n",
       "      <td>996</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>883</th>\n",
       "      <td>E997  LES OUT FORM GUERRA NAO CONVENCIONAL</td>\n",
       "      <td>997</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>884</th>\n",
       "      <td>E998  LES OP GUERRA OCORR APOS CESSACAO HOSTIL</td>\n",
       "      <td>998</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>885</th>\n",
       "      <td>E999  EFEIT TARDIOS DE LES OPERACAO DE GUERRA</td>\n",
       "      <td>999</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>886 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                          DESCRICAO CAUSAS\n",
       "0                                       001  COLERA    001\n",
       "1                 002  FEBRES TIFOIDE E PARATIFOIDE    002\n",
       "2              003  OUTRAS INFECCOES POR SALMONELLA    003\n",
       "3                                   004  SHIGUELOSE    004\n",
       "4        005  OUT INTOXIC ALIMENTARES (BACTERIANAS)    005\n",
       "..                                              ...    ...\n",
       "881  E995  LES OUT MEIOS OU N ESP OP GUERRA CONVENC    995\n",
       "882   E996  LES ARMA NUCLEAR EM OPERACOES DE GUERRA    996\n",
       "883      E997  LES OUT FORM GUERRA NAO CONVENCIONAL    997\n",
       "884  E998  LES OP GUERRA OCORR APOS CESSACAO HOSTIL    998\n",
       "885   E999  EFEIT TARDIOS DE LES OPERACAO DE GUERRA    999\n",
       "\n",
       "[886 rows x 2 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_CID9_table()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-09-15 18:51:51.586 | DEBUG    | pysus.online_data.SIM:get_CID10_table:93 - Stablishing connection with ftp.datasus.gov.br.\n",
      "220 Microsoft FTP Service\n",
      "2023-09-15 18:51:51.608 | DEBUG    | pysus.online_data.SIM:get_CID10_table:97 - Changing FTP work dir to: /dissemin/publicos/SIM/CID10/TABELAS\n",
      "2023-09-15 18:51:51.609 | INFO     | pysus.online_data.SIM:get_CID10_table:107 - Local parquet file found at /home/bida/pysus/SIM_CID10_.parquet\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>CID10</th>\n",
       "      <th>OPC</th>\n",
       "      <th>CAT</th>\n",
       "      <th>SUBCAT</th>\n",
       "      <th>DESCR</th>\n",
       "      <th>RESTRSEXO</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>A00</td>\n",
       "      <td></td>\n",
       "      <td>S</td>\n",
       "      <td>N</td>\n",
       "      <td>A00   Colera</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>A000</td>\n",
       "      <td></td>\n",
       "      <td>N</td>\n",
       "      <td>S</td>\n",
       "      <td>A00.0 Colera dev Vibrio cholerae 01 biot cholerae</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>A001</td>\n",
       "      <td></td>\n",
       "      <td>N</td>\n",
       "      <td>S</td>\n",
       "      <td>A00.1 Colera dev Vibrio cholerae 01 biot El Tor</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>A009</td>\n",
       "      <td></td>\n",
       "      <td>N</td>\n",
       "      <td>S</td>\n",
       "      <td>A00.9 Colera NE</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>A01</td>\n",
       "      <td></td>\n",
       "      <td>S</td>\n",
       "      <td>N</td>\n",
       "      <td>A01   Febres tifoide e paratifoide</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14252</th>\n",
       "      <td>Z926</td>\n",
       "      <td></td>\n",
       "      <td>N</td>\n",
       "      <td>S</td>\n",
       "      <td>Z92.6 História pessoal de quimioterapia para d...</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14253</th>\n",
       "      <td>U070</td>\n",
       "      <td></td>\n",
       "      <td>N</td>\n",
       "      <td>S</td>\n",
       "      <td>U07.0 Doença por cigarro eletrônico</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14254</th>\n",
       "      <td>U071</td>\n",
       "      <td></td>\n",
       "      <td>N</td>\n",
       "      <td>S</td>\n",
       "      <td>U07.1 Infecção pelo novo Coronavírus (COVID-19)</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14255</th>\n",
       "      <td>U072</td>\n",
       "      <td></td>\n",
       "      <td>N</td>\n",
       "      <td>S</td>\n",
       "      <td>U07.2 COVID-19, vírus não identificado</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14256</th>\n",
       "      <td>U07</td>\n",
       "      <td></td>\n",
       "      <td>S</td>\n",
       "      <td>N</td>\n",
       "      <td>U07 Uso emergencial do U07</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>14257 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      CID10 OPC CAT SUBCAT                                              DESCR  \\\n",
       "0       A00       S      N                                       A00   Colera   \n",
       "1      A000       N      S  A00.0 Colera dev Vibrio cholerae 01 biot cholerae   \n",
       "2      A001       N      S    A00.1 Colera dev Vibrio cholerae 01 biot El Tor   \n",
       "3      A009       N      S                                    A00.9 Colera NE   \n",
       "4       A01       S      N                 A01   Febres tifoide e paratifoide   \n",
       "...     ...  ..  ..    ...                                                ...   \n",
       "14252  Z926       N      S  Z92.6 História pessoal de quimioterapia para d...   \n",
       "14253  U070       N      S                U07.0 Doença por cigarro eletrônico   \n",
       "14254  U071       N      S    U07.1 Infecção pelo novo Coronavírus (COVID-19)   \n",
       "14255  U072       N      S             U07.2 COVID-19, vírus não identificado   \n",
       "14256   U07       S      N                         U07 Uso emergencial do U07   \n",
       "\n",
       "      RESTRSEXO  \n",
       "0             5  \n",
       "1             5  \n",
       "2             5  \n",
       "3             5  \n",
       "4             5  \n",
       "...         ...  \n",
       "14252         5  \n",
       "14253         5  \n",
       "14254         5  \n",
       "14255         5  \n",
       "14256         5  \n",
       "\n",
       "[14257 rows x 6 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_CID10_table()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-09-15 18:51:51.744 | DEBUG    | pysus.online_data.SIM:get_municipios:185 - Stablishing connection with ftp.datasus.gov.br.\n",
      "220 Microsoft FTP Service\n",
      "2023-09-15 18:51:51.768 | DEBUG    | pysus.online_data.SIM:get_municipios:189 - Changing FTP work dir to: /dissemin/publicos/SIM/CID10/TABELAS\n",
      "2023-09-15 18:51:51.769 | INFO     | pysus.online_data.SIM:get_municipios:199 - Local parquet file found at /home/bida/pysus/SIM_CADMUN_.parquet\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MUNCOD</th>\n",
       "      <th>MUNCODDV</th>\n",
       "      <th>SITUACAO</th>\n",
       "      <th>MUNSINP</th>\n",
       "      <th>MUNSIAFI</th>\n",
       "      <th>MUNNOME</th>\n",
       "      <th>MUNNOMEX</th>\n",
       "      <th>OBSERV</th>\n",
       "      <th>MUNSINON</th>\n",
       "      <th>MUNSINONDV</th>\n",
       "      <th>...</th>\n",
       "      <th>CSAUDCOD</th>\n",
       "      <th>RMETRCOD</th>\n",
       "      <th>AGLCOD</th>\n",
       "      <th>ANOINST</th>\n",
       "      <th>ANOEXT</th>\n",
       "      <th>SUCESSOR</th>\n",
       "      <th>LATITUDE</th>\n",
       "      <th>LONGITUDE</th>\n",
       "      <th>ALTITUDE</th>\n",
       "      <th>AREA</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>000000</td>\n",
       "      <td>0000000</td>\n",
       "      <td>IGNOR</td>\n",
       "      <td>00000</td>\n",
       "      <td></td>\n",
       "      <td>Ignorado ou exterior</td>\n",
       "      <td>IGNORADO OU EXTERIOR</td>\n",
       "      <td>Ignorado</td>\n",
       "      <td>,      ,000001-009999</td>\n",
       "      <td>,       ,0000001-0099999</td>\n",
       "      <td>...</td>\n",
       "      <td>00000</td>\n",
       "      <td>0000</td>\n",
       "      <td>0000</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>110000</td>\n",
       "      <td>1100000</td>\n",
       "      <td>IGNOR</td>\n",
       "      <td>26000</td>\n",
       "      <td></td>\n",
       "      <td>Munic¡pio ignorado - RO</td>\n",
       "      <td>MUNICIPIO IGNORADO - RO</td>\n",
       "      <td></td>\n",
       "      <td>,119999</td>\n",
       "      <td>,1199999</td>\n",
       "      <td>...</td>\n",
       "      <td>11000</td>\n",
       "      <td>1100</td>\n",
       "      <td>1100</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>110001</td>\n",
       "      <td>1100015</td>\n",
       "      <td>ATIVO</td>\n",
       "      <td>26016</td>\n",
       "      <td>0033</td>\n",
       "      <td>Alta Floresta D'Oeste</td>\n",
       "      <td>ALTA FLORESTA D'OESTE</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>11900</td>\n",
       "      <td>1190</td>\n",
       "      <td>1190</td>\n",
       "      <td>1986</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>-11.929</td>\n",
       "      <td>-61.996</td>\n",
       "      <td>350.0</td>\n",
       "      <td>7066.702</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>110002</td>\n",
       "      <td>1100023</td>\n",
       "      <td>ATIVO</td>\n",
       "      <td>26004</td>\n",
       "      <td>0007</td>\n",
       "      <td>Ariquemes</td>\n",
       "      <td>ARIQUEMES</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>11900</td>\n",
       "      <td>1190</td>\n",
       "      <td>1190</td>\n",
       "      <td>1977</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>-9.913</td>\n",
       "      <td>-63.041</td>\n",
       "      <td>142.0</td>\n",
       "      <td>4426.558</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>110003</td>\n",
       "      <td>1100031</td>\n",
       "      <td>ATIVO</td>\n",
       "      <td>26020</td>\n",
       "      <td>0037</td>\n",
       "      <td>Cabixi</td>\n",
       "      <td>CABIXI</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>11900</td>\n",
       "      <td>1190</td>\n",
       "      <td>1190</td>\n",
       "      <td>1989</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>-13.492</td>\n",
       "      <td>-60.545</td>\n",
       "      <td>230.0</td>\n",
       "      <td>1314.355</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5647</th>\n",
       "      <td>522208</td>\n",
       "      <td>5222088</td>\n",
       "      <td>TRANS</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>Wanderlndia (transf. p/TO)</td>\n",
       "      <td>WANDERLANDIA (TRANSF. P/TO)</td>\n",
       "      <td>Transferido para Tocantins</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>52980</td>\n",
       "      <td>5298</td>\n",
       "      <td>5298</td>\n",
       "      <td></td>\n",
       "      <td>1988</td>\n",
       "      <td>1722081</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5648</th>\n",
       "      <td>522210</td>\n",
       "      <td>5222104</td>\n",
       "      <td>TRANS</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>Xambio  (transf. p/TO)</td>\n",
       "      <td>XAMBIOA (TRANSF. P/TO)</td>\n",
       "      <td>Transferido para Tocantins</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>52980</td>\n",
       "      <td>5298</td>\n",
       "      <td>5298</td>\n",
       "      <td></td>\n",
       "      <td>1988</td>\n",
       "      <td>1722107</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5649</th>\n",
       "      <td>522220</td>\n",
       "      <td>5222203</td>\n",
       "      <td>ATIVO</td>\n",
       "      <td>08313</td>\n",
       "      <td>0067</td>\n",
       "      <td>Vila Boa</td>\n",
       "      <td>VILA BOA</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>52900</td>\n",
       "      <td>5301</td>\n",
       "      <td>5301</td>\n",
       "      <td>1993</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>-15.038</td>\n",
       "      <td>-47.059</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1060.170</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5650</th>\n",
       "      <td>522230</td>\n",
       "      <td>5222302</td>\n",
       "      <td>ATIVO</td>\n",
       "      <td>08323</td>\n",
       "      <td>1068</td>\n",
       "      <td>Vila Prop¡cio</td>\n",
       "      <td>VILA PROPICIO</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>52900</td>\n",
       "      <td>5290</td>\n",
       "      <td>5290</td>\n",
       "      <td>1997</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>-15.457</td>\n",
       "      <td>-48.889</td>\n",
       "      <td>744.0</td>\n",
       "      <td>2181.575</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5651</th>\n",
       "      <td>530010</td>\n",
       "      <td>5300108</td>\n",
       "      <td>ATIVO</td>\n",
       "      <td>23001</td>\n",
       "      <td>9701</td>\n",
       "      <td>Bras¡lia</td>\n",
       "      <td>BRASILIA</td>\n",
       "      <td></td>\n",
       "      <td>,530000-530009,530011-539999</td>\n",
       "      <td>,5300000-5300099,5300110-5399999</td>\n",
       "      <td>...</td>\n",
       "      <td>53001</td>\n",
       "      <td>5301</td>\n",
       "      <td>5301</td>\n",
       "      <td>1960</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>-15.780</td>\n",
       "      <td>-47.930</td>\n",
       "      <td>1171.0</td>\n",
       "      <td>5801.937</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5652 rows × 28 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      MUNCOD MUNCODDV SITUACAO MUNSINP MUNSIAFI                      MUNNOME  \\\n",
       "0     000000  0000000    IGNOR   00000                  Ignorado ou exterior   \n",
       "1     110000  1100000    IGNOR   26000               Munic¡pio ignorado - RO   \n",
       "2     110001  1100015    ATIVO   26016     0033        Alta Floresta D'Oeste   \n",
       "3     110002  1100023    ATIVO   26004     0007                    Ariquemes   \n",
       "4     110003  1100031    ATIVO   26020     0037                       Cabixi   \n",
       "...      ...      ...      ...     ...      ...                          ...   \n",
       "5647  522208  5222088    TRANS                   Wanderlndia (transf. p/TO)   \n",
       "5648  522210  5222104    TRANS                        Xambio  (transf. p/TO)   \n",
       "5649  522220  5222203    ATIVO   08313     0067                     Vila Boa   \n",
       "5650  522230  5222302    ATIVO   08323     1068                Vila Prop¡cio   \n",
       "5651  530010  5300108    ATIVO   23001     9701                     Bras¡lia   \n",
       "\n",
       "                         MUNNOMEX                      OBSERV  \\\n",
       "0            IGNORADO OU EXTERIOR                    Ignorado   \n",
       "1         MUNICIPIO IGNORADO - RO                               \n",
       "2           ALTA FLORESTA D'OESTE                               \n",
       "3                       ARIQUEMES                               \n",
       "4                          CABIXI                               \n",
       "...                           ...                         ...   \n",
       "5647  WANDERLANDIA (TRANSF. P/TO)  Transferido para Tocantins   \n",
       "5648       XAMBIOA (TRANSF. P/TO)  Transferido para Tocantins   \n",
       "5649                     VILA BOA                               \n",
       "5650                VILA PROPICIO                               \n",
       "5651                     BRASILIA                               \n",
       "\n",
       "                          MUNSINON                        MUNSINONDV  ...  \\\n",
       "0            ,      ,000001-009999          ,       ,0000001-0099999  ...   \n",
       "1                          ,119999                          ,1199999  ...   \n",
       "2                                                                     ...   \n",
       "3                                                                     ...   \n",
       "4                                                                     ...   \n",
       "...                            ...                               ...  ...   \n",
       "5647                                                                  ...   \n",
       "5648                                                                  ...   \n",
       "5649                                                                  ...   \n",
       "5650                                                                  ...   \n",
       "5651  ,530000-530009,530011-539999  ,5300000-5300099,5300110-5399999  ...   \n",
       "\n",
       "     CSAUDCOD RMETRCOD AGLCOD ANOINST ANOEXT SUCESSOR LATITUDE LONGITUDE  \\\n",
       "0       00000     0000   0000                            0.000     0.000   \n",
       "1       11000     1100   1100                            0.000     0.000   \n",
       "2       11900     1190   1190    1986                  -11.929   -61.996   \n",
       "3       11900     1190   1190    1977                   -9.913   -63.041   \n",
       "4       11900     1190   1190    1989                  -13.492   -60.545   \n",
       "...       ...      ...    ...     ...    ...      ...      ...       ...   \n",
       "5647    52980     5298   5298           1988  1722081    0.000     0.000   \n",
       "5648    52980     5298   5298           1988  1722107    0.000     0.000   \n",
       "5649    52900     5301   5301    1993                  -15.038   -47.059   \n",
       "5650    52900     5290   5290    1997                  -15.457   -48.889   \n",
       "5651    53001     5301   5301    1960                  -15.780   -47.930   \n",
       "\n",
       "     ALTITUDE      AREA  \n",
       "0         0.0     0.000  \n",
       "1         0.0     0.000  \n",
       "2       350.0  7066.702  \n",
       "3       142.0  4426.558  \n",
       "4       230.0  1314.355  \n",
       "...       ...       ...  \n",
       "5647      0.0     0.000  \n",
       "5648      0.0     0.000  \n",
       "5649      0.0  1060.170  \n",
       "5650    744.0  2181.575  \n",
       "5651   1171.0  5801.937  \n",
       "\n",
       "[5652 rows x 28 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_municipios()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-09-15 18:51:51.942 | DEBUG    | pysus.online_data.SIM:get_ocupations:231 - Stablishing connection with ftp.datasus.gov.br.\n",
      "220 Microsoft FTP Service\n",
      "2023-09-15 18:51:51.965 | DEBUG    | pysus.online_data.SIM:get_ocupations:235 - Changing FTP work dir to: /dissemin/publicos/SIM/CID10/TABELAS\n",
      "2023-09-15 18:51:51.966 | INFO     | pysus.online_data.SIM:get_ocupations:244 - Local parquet file found at /home/bida/pysus/SIM_TABOCUP_.parquet\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>CODIGO</th>\n",
       "      <th>DESCRICAO</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>639</td>\n",
       "      <td>ABALIZADOR (AREAS AGRICOLAS)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>759</td>\n",
       "      <td>ABASTECEDOR DE ESPULAS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>893</td>\n",
       "      <td>ABASTECEDOR DE FORNO (CERAMICA)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>999</td>\n",
       "      <td>ABASTECEDOR DE LINHA DE PRODUCAO</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>739</td>\n",
       "      <td>ABASTECEDOR DE MAQUINAS (MAD. E PASTA PAPEL)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3559</th>\n",
       "      <td>540</td>\n",
       "      <td>ZELADOR DE VESTIARIOS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3560</th>\n",
       "      <td>551</td>\n",
       "      <td>ZELADOR SEM ESPECIFICACAO</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3561</th>\n",
       "      <td>728</td>\n",
       "      <td>ZINCADOR</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3562</th>\n",
       "      <td>051</td>\n",
       "      <td>ZOOLOGO</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3563</th>\n",
       "      <td>065</td>\n",
       "      <td>ZOOTECNISTA</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3564 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     CODIGO                                     DESCRICAO\n",
       "0       639                  ABALIZADOR (AREAS AGRICOLAS)\n",
       "1       759                        ABASTECEDOR DE ESPULAS\n",
       "2       893               ABASTECEDOR DE FORNO (CERAMICA)\n",
       "3       999              ABASTECEDOR DE LINHA DE PRODUCAO\n",
       "4       739  ABASTECEDOR DE MAQUINAS (MAD. E PASTA PAPEL)\n",
       "...     ...                                           ...\n",
       "3559    540                         ZELADOR DE VESTIARIOS\n",
       "3560    551                     ZELADOR SEM ESPECIFICACAO\n",
       "3561    728                                      ZINCADOR\n",
       "3562    051                                       ZOOLOGO\n",
       "3563    065                                   ZOOTECNISTA\n",
       "\n",
       "[3564 rows x 2 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_ocupations()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preprocessing SIM data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pysus.ftp.databases.sim import SIM\n",
    "from pysus.preprocessing.decoders import translate_variables_SIM\n",
    "from pysus.preprocessing.SIM import group_and_count, redistribute_missing, redistribute_cid_chapter\n",
    "\n",
    "sim = SIM().load()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "DOAC2010.parquet:   0%|                                                                                          | 0.00/7.37k [00:00<?, ?B/s]/home/bida/micromamba/envs/pysus/lib/python3.11/site-packages/tqdm/std.py:533: TqdmWarning: clamping frac to range [0, 1]\n",
      "  full_bar = Bar(frac,\n",
      "DOAC2010.parquet: 100%|█████████████████████████████████████████████████████████████████████████████████| 7.37k/7.37k [00:00<00:00, 34.4kB/s]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>contador</th>\n",
       "      <th>ORIGEM</th>\n",
       "      <th>TIPOBITO</th>\n",
       "      <th>DTOBITO</th>\n",
       "      <th>HORAOBITO</th>\n",
       "      <th>NATURAL</th>\n",
       "      <th>DTNASC</th>\n",
       "      <th>IDADE</th>\n",
       "      <th>SEXO</th>\n",
       "      <th>RACACOR</th>\n",
       "      <th>...</th>\n",
       "      <th>DTCADASTRO</th>\n",
       "      <th>ATESTANTE</th>\n",
       "      <th>FONTEINV</th>\n",
       "      <th>DTRECEBIM</th>\n",
       "      <th>UFINFORM</th>\n",
       "      <th>CB_PRE</th>\n",
       "      <th>MORTEPARTO</th>\n",
       "      <th>DTCADINF</th>\n",
       "      <th>TPOBITOCOR</th>\n",
       "      <th>DTCADINV</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>06082010</td>\n",
       "      <td>2000</td>\n",
       "      <td>831</td>\n",
       "      <td>09041945</td>\n",
       "      <td>465</td>\n",
       "      <td>2</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>19082010</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>20092010</td>\n",
       "      <td></td>\n",
       "      <td>R98</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>06082010</td>\n",
       "      <td>1300</td>\n",
       "      <td>812</td>\n",
       "      <td>20011912</td>\n",
       "      <td>498</td>\n",
       "      <td>1</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>19082010</td>\n",
       "      <td></td>\n",
       "      <td>2</td>\n",
       "      <td>01102010</td>\n",
       "      <td></td>\n",
       "      <td>R98</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>02102010</td>\n",
       "      <td>1700</td>\n",
       "      <td>812</td>\n",
       "      <td>17032010</td>\n",
       "      <td>306</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>...</td>\n",
       "      <td>05012011</td>\n",
       "      <td>3</td>\n",
       "      <td></td>\n",
       "      <td>10022011</td>\n",
       "      <td></td>\n",
       "      <td>R98</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>07042010</td>\n",
       "      <td>2300</td>\n",
       "      <td></td>\n",
       "      <td>07042010</td>\n",
       "      <td>005</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>06052010</td>\n",
       "      <td>1</td>\n",
       "      <td></td>\n",
       "      <td>20072010</td>\n",
       "      <td></td>\n",
       "      <td>O689</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>13052010</td>\n",
       "      <td>0030</td>\n",
       "      <td>812</td>\n",
       "      <td>04081971</td>\n",
       "      <td>438</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>21062010</td>\n",
       "      <td>5</td>\n",
       "      <td></td>\n",
       "      <td>20072010</td>\n",
       "      <td></td>\n",
       "      <td>X999</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3004</th>\n",
       "      <td>3005</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>28092010</td>\n",
       "      <td>0100</td>\n",
       "      <td></td>\n",
       "      <td>20041958</td>\n",
       "      <td>452</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>...</td>\n",
       "      <td>20102010</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>11112010</td>\n",
       "      <td></td>\n",
       "      <td>I619</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3005</th>\n",
       "      <td>3006</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>14102010</td>\n",
       "      <td>1015</td>\n",
       "      <td></td>\n",
       "      <td>24031928</td>\n",
       "      <td>482</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>06122010</td>\n",
       "      <td>2</td>\n",
       "      <td></td>\n",
       "      <td>19012011</td>\n",
       "      <td></td>\n",
       "      <td>I219</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3006</th>\n",
       "      <td>3007</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>21112010</td>\n",
       "      <td>0650</td>\n",
       "      <td></td>\n",
       "      <td>11121955</td>\n",
       "      <td>454</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>09122010</td>\n",
       "      <td>5</td>\n",
       "      <td></td>\n",
       "      <td>17012011</td>\n",
       "      <td></td>\n",
       "      <td>C349</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3007</th>\n",
       "      <td>3008</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>10112010</td>\n",
       "      <td>0500</td>\n",
       "      <td></td>\n",
       "      <td>07071955</td>\n",
       "      <td>455</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>04012011</td>\n",
       "      <td>2</td>\n",
       "      <td></td>\n",
       "      <td>19012011</td>\n",
       "      <td></td>\n",
       "      <td>C787</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3008</th>\n",
       "      <td>3009</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>10102010</td>\n",
       "      <td>1240</td>\n",
       "      <td></td>\n",
       "      <td>07051976</td>\n",
       "      <td>434</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>...</td>\n",
       "      <td>28122010</td>\n",
       "      <td>2</td>\n",
       "      <td></td>\n",
       "      <td>26112010</td>\n",
       "      <td></td>\n",
       "      <td>I739</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3009 rows × 59 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      contador  ORIGEM  TIPOBITO   DTOBITO  HORAOBITO  NATURAL    DTNASC  \\\n",
       "0     1         1       2         06082010  2000       831      09041945   \n",
       "1     2         1       2         06082010  1300       812      20011912   \n",
       "2     3         1       2         02102010  1700       812      17032010   \n",
       "3     4         1       2         07042010  2300                07042010   \n",
       "4     5         1       2         13052010  0030       812      04081971   \n",
       "...        ...     ...       ...       ...        ...      ...       ...   \n",
       "3004  3005      1       2         28092010  0100                20041958   \n",
       "3005  3006      1       2         14102010  1015                24031928   \n",
       "3006  3007      1       2         21112010  0650                11121955   \n",
       "3007  3008      1       2         10112010  0500                07071955   \n",
       "3008  3009      1       2         10102010  1240                07051976   \n",
       "\n",
       "      IDADE  SEXO  RACACOR  ...  DTCADASTRO  ATESTANTE  FONTEINV  DTRECEBIM  \\\n",
       "0     465       2           ...  19082010                         20092010    \n",
       "1     498       1           ...  19082010               2         01102010    \n",
       "2     306       2  4        ...  05012011    3                    10022011    \n",
       "3     005       2  1        ...  06052010    1                    20072010    \n",
       "4     438       1  1        ...  21062010    5                    20072010    \n",
       "...     ...   ...      ...  ...         ...        ...       ...        ...   \n",
       "3004  452       2  4        ...  20102010                         11112010    \n",
       "3005  482       1  1        ...  06122010    2                    19012011    \n",
       "3006  454       2  1        ...  09122010    5                    17012011    \n",
       "3007  455       1  1        ...  04012011    2                    19012011    \n",
       "3008  434       2  4        ...  28122010    2                    26112010    \n",
       "\n",
       "     UFINFORM  CB_PRE MORTEPARTO DTCADINF TPOBITOCOR DTCADINV  \n",
       "0              R98                                             \n",
       "1              R98                                             \n",
       "2              R98                                             \n",
       "3              O689                                            \n",
       "4              X999                                            \n",
       "...       ...     ...        ...      ...        ...      ...  \n",
       "3004           I619                                            \n",
       "3005           I219                                            \n",
       "3006           C349                                            \n",
       "3007           C787                                            \n",
       "3008           I739                                            \n",
       "\n",
       "[3009 rows x 59 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = sim.download(sim.get_files(\"CID10\",'ac',2010)).to_dataframe()\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['SEXO'] = df['SEXO'].astype('object')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-09-19 13:55:09.529 | DEBUG    | pysus.online_data.SIM:get_municipios:185 - Stablishing connection with ftp.datasus.gov.br.\n",
      "220 Microsoft FTP Service\n",
      "2023-09-19 13:55:09.552 | DEBUG    | pysus.online_data.SIM:get_municipios:189 - Changing FTP work dir to: /dissemin/publicos/SIM/CID10/TABELAS\n",
      "2023-09-19 13:55:09.553 | INFO     | pysus.online_data.SIM:get_municipios:199 - Local parquet file found at /home/bida/pysus/SIM_CADMUN_.parquet\n",
      "/home/bida/Projetos/InfoDengue/PySUS/pysus/preprocessing/decoders.py:122: FutureWarning: The series.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  return df[\"MUNCODDV\"].append(df[\"MUNCOD\"]).astype(\"int64\").values\n",
      "2023-09-19 13:55:09.745 | DEBUG    | pysus.online_data.SIM:get_CID10_chapters_table:47 - Stablishing connection with ftp.datasus.gov.br.\n",
      "220 Microsoft FTP Service\n",
      "2023-09-19 13:55:09.767 | DEBUG    | pysus.online_data.SIM:get_CID10_chapters_table:51 - Changing FTP work dir to: /dissemin/publicos/SIM/CID10/TABELAS\n",
      "2023-09-19 13:55:09.768 | INFO     | pysus.online_data.SIM:get_CID10_chapters_table:61 - Local parquet file found at /home/bida/pysus/SIM_CIDCAP10_.parquet\n"
     ]
    }
   ],
   "source": [
    "variables = ['CODMUNRES','SEXO','IDADE_ANOS','CID10_CHAPTER']\n",
    "\n",
    "df = translate_variables_SIM(\n",
    "    df,\n",
    "    age_classes=True,\n",
    "    classify_cid10_chapters=True,\n",
    ")\n",
    "df = df[variables]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "counts = group_and_count(df,variables)\n",
    "counts[\"COUNTS_ORIGINAL\"] = counts[\"COUNTS\"]\n",
    "nan_string = 'missing'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>CODMUNRES</th>\n",
       "      <th>SEXO</th>\n",
       "      <th>IDADE_ANOS</th>\n",
       "      <th>CID10_CHAPTER</th>\n",
       "      <th>COUNTS</th>\n",
       "      <th>COUNTS_ORIGINAL</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>1200013.0</td>\n",
       "      <td>Feminino</td>\n",
       "      <td>[0.0, 1.0)</td>\n",
       "      <td>16</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9397</th>\n",
       "      <td>1200104.0</td>\n",
       "      <td>Feminino</td>\n",
       "      <td>[0.0, 1.0)</td>\n",
       "      <td>16</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10615</th>\n",
       "      <td>1200104.0</td>\n",
       "      <td>Feminino</td>\n",
       "      <td>[72.0, 73.0)</td>\n",
       "      <td>10</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10961</th>\n",
       "      <td>1200104.0</td>\n",
       "      <td>Masculino</td>\n",
       "      <td>[0.0, 1.0)</td>\n",
       "      <td>16</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11423</th>\n",
       "      <td>1200104.0</td>\n",
       "      <td>Masculino</td>\n",
       "      <td>[27.0, 28.0)</td>\n",
       "      <td>20</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96787</th>\n",
       "      <td>1200708.0</td>\n",
       "      <td>Masculino</td>\n",
       "      <td>[81.0, 82.0)</td>\n",
       "      <td>9</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98539</th>\n",
       "      <td>1200807.0</td>\n",
       "      <td>Feminino</td>\n",
       "      <td>[0.0, 1.0)</td>\n",
       "      <td>10</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98545</th>\n",
       "      <td>1200807.0</td>\n",
       "      <td>Feminino</td>\n",
       "      <td>[0.0, 1.0)</td>\n",
       "      <td>16</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>101420</th>\n",
       "      <td>1200807.0</td>\n",
       "      <td>Masculino</td>\n",
       "      <td>[77.0, 78.0)</td>\n",
       "      <td>18</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>101539</th>\n",
       "      <td>1200807.0</td>\n",
       "      <td>Masculino</td>\n",
       "      <td>[84.0, 85.0)</td>\n",
       "      <td>18</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>466 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        CODMUNRES       SEXO    IDADE_ANOS CID10_CHAPTER  COUNTS  \\\n",
       "13      1200013.0   Feminino    [0.0, 1.0)            16     2.0   \n",
       "9397    1200104.0   Feminino    [0.0, 1.0)            16     2.0   \n",
       "10615   1200104.0   Feminino  [72.0, 73.0)            10     2.0   \n",
       "10961   1200104.0  Masculino    [0.0, 1.0)            16     3.0   \n",
       "11423   1200104.0  Masculino  [27.0, 28.0)            20     2.0   \n",
       "...           ...        ...           ...           ...     ...   \n",
       "96787   1200708.0  Masculino  [81.0, 82.0)             9     2.0   \n",
       "98539   1200807.0   Feminino    [0.0, 1.0)            10     2.0   \n",
       "98545   1200807.0   Feminino    [0.0, 1.0)            16     4.0   \n",
       "101420  1200807.0  Masculino  [77.0, 78.0)            18     2.0   \n",
       "101539  1200807.0  Masculino  [84.0, 85.0)            18     2.0   \n",
       "\n",
       "        COUNTS_ORIGINAL  \n",
       "13                  2.0  \n",
       "9397                2.0  \n",
       "10615               2.0  \n",
       "10961               3.0  \n",
       "11423               2.0  \n",
       "...                 ...  \n",
       "96787               2.0  \n",
       "98539               2.0  \n",
       "98545               4.0  \n",
       "101420              2.0  \n",
       "101539              2.0  \n",
       "\n",
       "[466 rows x 6 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "counts = redistribute_missing(counts,['CODMUNRES','SEXO','IDADE_ANOS'],nan_string=nan_string)\n",
    "counts[counts[\"COUNTS\"] > 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>CODMUNRES</th>\n",
       "      <th>SEXO</th>\n",
       "      <th>IDADE_ANOS</th>\n",
       "      <th>CID10_CHAPTER</th>\n",
       "      <th>COUNTS</th>\n",
       "      <th>COUNTS_ORIGINAL</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1200013.0</td>\n",
       "      <td>Feminino</td>\n",
       "      <td>[0.0, 1.0)</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>1200013.0</td>\n",
       "      <td>Feminino</td>\n",
       "      <td>[1.0, 2.0)</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>1200013.0</td>\n",
       "      <td>Feminino</td>\n",
       "      <td>[2.0, 3.0)</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51</th>\n",
       "      <td>1200013.0</td>\n",
       "      <td>Feminino</td>\n",
       "      <td>[3.0, 4.0)</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68</th>\n",
       "      <td>1200013.0</td>\n",
       "      <td>Feminino</td>\n",
       "      <td>[4.0, 5.0)</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>107831</th>\n",
       "      <td>NA</td>\n",
       "      <td>NA</td>\n",
       "      <td>[87.0, 88.0)</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>107848</th>\n",
       "      <td>NA</td>\n",
       "      <td>NA</td>\n",
       "      <td>[88.0, 89.0)</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>107865</th>\n",
       "      <td>NA</td>\n",
       "      <td>NA</td>\n",
       "      <td>[89.0, 90.0)</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>107882</th>\n",
       "      <td>NA</td>\n",
       "      <td>NA</td>\n",
       "      <td>[90.0, inf)</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>107899</th>\n",
       "      <td>NA</td>\n",
       "      <td>NA</td>\n",
       "      <td>NA</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>6348 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        CODMUNRES      SEXO    IDADE_ANOS CID10_CHAPTER  COUNTS  \\\n",
       "0       1200013.0  Feminino    [0.0, 1.0)             1     1.0   \n",
       "17      1200013.0  Feminino    [1.0, 2.0)             1     0.0   \n",
       "34      1200013.0  Feminino    [2.0, 3.0)             1     0.0   \n",
       "51      1200013.0  Feminino    [3.0, 4.0)             1     0.0   \n",
       "68      1200013.0  Feminino    [4.0, 5.0)             1     0.0   \n",
       "...           ...       ...           ...           ...     ...   \n",
       "107831         NA        NA  [87.0, 88.0)             1     0.0   \n",
       "107848         NA        NA  [88.0, 89.0)             1     0.0   \n",
       "107865         NA        NA  [89.0, 90.0)             1     0.0   \n",
       "107882         NA        NA   [90.0, inf)             1     0.0   \n",
       "107899         NA        NA            NA             1     0.0   \n",
       "\n",
       "        COUNTS_ORIGINAL  \n",
       "0                   1.0  \n",
       "17                  0.0  \n",
       "34                  0.0  \n",
       "51                  0.0  \n",
       "68                  0.0  \n",
       "...                 ...  \n",
       "107831              0.0  \n",
       "107848              0.0  \n",
       "107865              0.0  \n",
       "107882              0.0  \n",
       "107899              0.0  \n",
       "\n",
       "[6348 rows x 6 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "counts = redistribute_cid_chapter(counts,['CODMUNRES','SEXO','IDADE_ANOS'])\n",
    "counts[counts['CID10_CHAPTER'] == 1]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  },
  "vscode": {
   "interpreter": {
    "hash": "2a96a5ccec8dfcba7d06b2e71f6eef3b5dac5716461bf5d73ea1bb7ee462cdaa"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}